/* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "zlang_in.h"

TLS char	_zlang_token_type_str1[4+1] = "" ;
TLS char	_zlang_token_type_str2[4+1] = "" ;
TLS char	_zlang_token_type_str3[4+1] = "" ;
TLS char	_zlang_token_type_str[4+1] = "" ;

int LexicalAnalysisSourceFile( char *filename , struct ZlangRuntime *rt )
{
	char		*source_begin = NULL ;
	int		nret = 0;

	TEST_RUNTIME_DEBUG_THEN_PRINT( rt , "enter LexicalAnalysisSourceFile filename[%s]" , filename )
	
#if ( defined __linux ) || ( defined __unix )
	int		fd;
	struct stat	stat_buf;
	uint64_t	size ;

	fd = open( filename , O_RDONLY ) ;
	if( fd == -1 )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_OPEN , "source file[%s] not found" , filename )
		return ZLANG_ERROR_OPEN;
	}
	
	nret = fstat( fd , & stat_buf ) ;
	if( nret == -1 )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_STAT , "fstat[%s] failed" , filename )
		close( fd );
		return ZLANG_ERROR_OPEN;
	}
	
	size = stat_buf.st_size ;
	source_begin = mmap( NULL , size , PROT_READ , MAP_PRIVATE , fd , 0 ) ;
	if( source_begin == NULL )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_STAT , "mmap[%s] size[%ld] failed" , filename , size )
		close( fd );
		return ZLANG_ERROR_OPEN;
	}
	
	close( fd );
#elif ( defined _WIN32 )
	HANDLE		file ;
	HANDLE		mapping ;
	LARGE_INTEGER	size2 ;
	__int64		size ;

	file = CreateFile( filename , GENERIC_READ , FILE_SHARE_READ , NULL , OPEN_EXISTING , FILE_ATTRIBUTE_NORMAL , NULL );
	if( file == INVALID_HANDLE_VALUE )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_OPEN , "source file[%s] not found" , filename )
		return ZLANG_ERROR_OPEN;
	}

	GetFileSizeEx( file , & size2 );
	size = size2.QuadPart ;

	mapping = CreateFileMapping( file , NULL , PAGE_READONLY , 0 , 0 , NULL ) ;
	if( mapping == INVALID_HANDLE_VALUE )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_STAT , "mmap[%s] size[%I64d] failed" , filename , size )
		CloseHandle( file );
		return ZLANG_ERROR_OPEN;
	}

	source_begin = (char*)MapViewOfFile( mapping , FILE_MAP_READ , 0 , 0 , (size_t)size ) ;
	if( source_begin == NULL )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_STAT , "mmap[%s] size[%I64d] failed" , filename , size )
		CloseHandle( mapping );
		CloseHandle( file );
		return ZLANG_ERROR_OPEN;
	}
#endif
	
	nret = LexicalAnalysisSource( filename , source_begin , source_begin+size-1 , rt ) ;
	
#if ( defined __linux ) || ( defined __unix )
	munmap( source_begin , size );
#elif ( defined _WIN32 )
	UnmapViewOfFile( source_begin );
	CloseHandle( mapping );
	CloseHandle( file );
#endif

	TEST_RUNTIME_DEBUG_THEN_PRINT( rt , "leave LexicalAnalysisSourceFile filename[%s]" , filename )
	
	return nret;
}

int LexicalAnalysisSource( char *source_filename , char *source_begin , char *source_end , struct ZlangRuntime *rt )
{
	struct ZlangSourceFileDataPageHeader	*source_file_datapage = NULL ;
	char					*source_file_dataunit = NULL ;
	struct ZlangTokenDataUnitHeader		token_dataunit_header ;
	struct ZlangTokenDataPageHeader		*token_datapage = NULL ;
	char					*token_dataunit = NULL ;
	
	char					*token_begin = NULL ;
	unsigned char				charset_flag ;
	unsigned char				include_flag ;
	struct ZlangIncludeFile			*include_file = NULL ;
	
	int					nret = 0 ;
	
	TEST_RUNTIME_DEBUG_THEN_PRINT( rt , "enter LexicalAnalysisSource filename[%s] begin[%p]" , source_filename , source_begin )
	
	do
	{
		source_file_dataunit = AddAppendonlyDataUnit( rt->curr_source_file_datapage->datapage , NULL , source_filename , strlen(source_filename) ) ;
		if( source_file_dataunit == NULL )
		{
			char		*datapage = NULL ;
			
			datapage = CreateAppendonlyDataPage( ZLANG_SOURCE_FILE_DATAPAGE_MAGIC , ZLANG_SOURCE_FILE_DATAPAGE_SIZE , sizeof(struct ZlangSourceFileDataPageHeader) , 0 , 1 ) ;
			if( datapage == NULL )
			{
				SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_ALLOC , "create source-file appendonly-datapage failed" )
				return ZLANG_ERROR_ALLOC;
			}
			source_file_datapage = GetAppendonlyDataPageUserHeader(datapage) ;
			source_file_datapage->datapage = datapage ;
			list_add_tail( & (source_file_datapage->this_source_file_datapage) , & (rt->source_file_datapage_list) );
			rt->curr_source_file_datapage = source_file_datapage ;
		}
	}
	while( source_file_dataunit == NULL );
	
	charset_flag = 0 ;
	include_flag = 0 ;
	
	memset( & token_dataunit_header , 0x00 , sizeof(struct ZlangTokenDataUnitHeader) );
	token_dataunit_header.source_filename = source_filename ;
	token_dataunit_header.source_row = 1 ;
	token_dataunit_header.source_col = 1 ;
	for( ; ; )
	{
		token_dataunit_header.token_type = TokenWord( rt->charset , & source_begin , source_end , & token_begin , & (token_dataunit_header.token_len) , & (token_dataunit_header.source_row) , & (token_dataunit_header.source_col) ) ;
		if( token_dataunit_header.token_type == TOKEN_TYPE_ERROR )
		{
			SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_TOKEN , "token word error" )
			return ZLANG_ERROR_TOKEN;
		}
		else if( token_dataunit_header.token_type == TOKEN_TYPE_END_OF_SOURCE )
		{
			break;
		}
		
		if( token_dataunit_header.token_type == TOKEN_TYPE_IDENTIFICATION )
		{
			if( token_dataunit_header.token_len == 7 && token_begin[0] == 'c' && token_begin[1] == 'h' && token_begin[2] == 'a' && token_begin[3] == 'r' && token_begin[4] == 's' && token_begin[5] == 'e' && token_begin[6] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_CHARSET ;
			else if( token_dataunit_header.token_len == 1 && token_begin[0] == 'S' )
				token_dataunit_header.token_type = TOKEN_TYPE_SHORT_NUMBER ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'U' && token_begin[1] == 'S' )
				token_dataunit_header.token_type = TOKEN_TYPE_USHORT_NUMBER ;
			else if( token_dataunit_header.token_len == 1 && token_begin[0] == 'I' )
				token_dataunit_header.token_type = TOKEN_TYPE_INT_NUMBER ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'U' && token_begin[1] == 'I' )
				token_dataunit_header.token_type = TOKEN_TYPE_UINT_NUMBER ;
			else if( token_dataunit_header.token_len == 1 && token_begin[0] == 'L' )
				token_dataunit_header.token_type = TOKEN_TYPE_LONG_NUMBER ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'U' && token_begin[1] == 'L' )
				token_dataunit_header.token_type = TOKEN_TYPE_ULONG_NUMBER ;
			else if( token_dataunit_header.token_len == 1 && token_begin[0] == 'F' )
				token_dataunit_header.token_type = TOKEN_TYPE_SINGLE_FLOAT_NUMBER ;
			else if( token_dataunit_header.token_len == 1 && token_begin[0] == 'D' )
				token_dataunit_header.token_type = TOKEN_TYPE_DOUBLE_FLOAT_NUMBER ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 't' && token_begin[1] == 'r' && token_begin[2] == 'u' && token_begin[3] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_TRUE ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'f' && token_begin[1] == 'a' && token_begin[2] == 'l' && token_begin[3] == 's' && token_begin[4] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_FALSE ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 'n' && token_begin[1] == 'u' && token_begin[2] == 'l' && token_begin[3] == 'l' )
				token_dataunit_header.token_type = TOKEN_TYPE_NULL ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'i' && token_begin[1] == 'n' && token_begin[2] == 'c' && token_begin[3] == 'l' && token_begin[4] == 'u' && token_begin[5] == 'd' && token_begin[6] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_INCLUDE ;
			else if( token_dataunit_header.token_len == 3 && token_begin[0] == 'n' && token_begin[1] == 'e' && token_begin[2] == 'w' )
				token_dataunit_header.token_type = TOKEN_TYPE_NEW ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'i' && token_begin[1] == 'f' )
				token_dataunit_header.token_type = TOKEN_TYPE_IF ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 'e' && token_begin[1] == 'l' && token_begin[2] == 's' && token_begin[3] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_ELSE ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 's' && token_begin[1] == 'w' && token_begin[2] == 'i' && token_begin[3] == 't' && token_begin[4] == 'c' && token_begin[5] == 'h' )
				token_dataunit_header.token_type = TOKEN_TYPE_SWITCH ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 'c' && token_begin[1] == 'a' && token_begin[2] == 's' && token_begin[3] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_CASE ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'd' && token_begin[1] == 'e' && token_begin[2] == 'f' && token_begin[3] == 'a' && token_begin[4] == 'u' && token_begin[5] == 'l' && token_begin[6] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_DEFAULT ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'd' && token_begin[1] == 'o' )
				token_dataunit_header.token_type = TOKEN_TYPE_DO ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'w' && token_begin[1] == 'h' && token_begin[2] == 'i' && token_begin[3] == 'l' && token_begin[4] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_WHILE ;
			else if( token_dataunit_header.token_len == 3 && token_begin[0] == 'f' && token_begin[1] == 'o' && token_begin[2] == 'r' )
				token_dataunit_header.token_type = TOKEN_TYPE_FOR ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 'i' && token_begin[1] == 'n' )
				token_dataunit_header.token_type = TOKEN_TYPE_IN ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 'f' && token_begin[1] == 'r' && token_begin[2] == 'o' && token_begin[3] == 'm' )
				token_dataunit_header.token_type = TOKEN_TYPE_FROM ;
			else if( token_dataunit_header.token_len == 2 && token_begin[0] == 't' && token_begin[1] == 'o' )
				token_dataunit_header.token_type = TOKEN_TYPE_TO ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'f' && token_begin[1] == 'o' && token_begin[2] == 'r' && token_begin[3] == 'e' && token_begin[4] == 'a' && token_begin[5] == 'c' && token_begin[6] == 'h' )
				token_dataunit_header.token_type = TOKEN_TYPE_FOREACH ;
			else if( token_dataunit_header.token_len == 8 && token_begin[0] == 'c' && token_begin[1] == 'o' && token_begin[2] == 'n' && token_begin[3] == 't' && token_begin[4] == 'i' && token_begin[5] == 'n' && token_begin[6] == 'u' && token_begin[7] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_CONTINUE ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'b' && token_begin[1] == 'r' && token_begin[2] == 'e' && token_begin[3] == 'a' && token_begin[4] == 'k' )
				token_dataunit_header.token_type = TOKEN_TYPE_BREAK ;
			else if( token_dataunit_header.token_len == 3 && token_begin[0] == 't' && token_begin[1] == 'r' && token_begin[2] == 'y' )
				token_dataunit_header.token_type = TOKEN_TYPE_TRY ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 't' && token_begin[1] == 'h' && token_begin[2] == 'r' && token_begin[3] == 'o' && token_begin[4] == 'w' )
				token_dataunit_header.token_type = TOKEN_TYPE_THROW ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'c' && token_begin[1] == 'a' && token_begin[2] == 't' && token_begin[3] == 'c' && token_begin[4] == 'h' )
				token_dataunit_header.token_type = TOKEN_TYPE_CATCH ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'f' && token_begin[1] == 'i' && token_begin[2] == 'n' && token_begin[3] == 'a' && token_begin[4] == 'l' && token_begin[5] == 'l' && token_begin[6] == 'y' )
				token_dataunit_header.token_type = TOKEN_TYPE_FINALLY ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'u' && token_begin[1] == 'n' && token_begin[2] == 'c' && token_begin[3] == 'a' && token_begin[4] == 't' && token_begin[5] == 'c' && token_begin[6] == 'h' )
				token_dataunit_header.token_type = TOKEN_TYPE_UNCATCH ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'i' && token_begin[1] == 'm' && token_begin[2] == 'p' && token_begin[3] == 'o' && token_begin[4] == 'r' && token_begin[5] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_IMPORT ;
			else if( token_dataunit_header.token_len == 8 && token_begin[0] == 'f' && token_begin[1] == 'u' && token_begin[2] == 'n' && token_begin[3] == 'c' && token_begin[4] == 't' && token_begin[5] == 'i' && token_begin[6] == 'o' && token_begin[7] == 'n' )
				token_dataunit_header.token_type = TOKEN_TYPE_FUNCTION ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'o' && token_begin[1] == 'b' && token_begin[2] == 'j' && token_begin[3] == 'e' && token_begin[4] == 'c' && token_begin[5] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_OBJECT ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'e' && token_begin[1] == 'x' && token_begin[2] == 't' && token_begin[3] == 'e' && token_begin[4] == 'n' && token_begin[5] == 'd' && token_begin[6] == 's' )
				token_dataunit_header.token_type = TOKEN_TYPE_EXTENDS ;
			else if( token_dataunit_header.token_len == 9 && token_begin[0] == 'i' && token_begin[1] == 'n' && token_begin[2] == 't' && token_begin[3] == 'e' && token_begin[4] == 'r' && token_begin[5] == 'f' && token_begin[6] == 'a' && token_begin[7] == 'c' && token_begin[8] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_INTERFACE ;
			else if( token_dataunit_header.token_len == 10 && token_begin[0] == 'i' && token_begin[1] == 'm' && token_begin[2] == 'p' && token_begin[3] == 'l' && token_begin[4] == 'e' && token_begin[5] == 'm' && token_begin[6] == 'e' && token_begin[7] == 'n' && token_begin[8] == 't' && token_begin[9] == 's' )
				token_dataunit_header.token_type = TOKEN_TYPE_IMPLEMENTS ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'p' && token_begin[1] == 'u' && token_begin[2] == 'b' && token_begin[3] == 'l' && token_begin[4] == 'i' && token_begin[5] == 'c' )
				token_dataunit_header.token_type = TOKEN_TYPE_PUBLIC ;
			else if( token_dataunit_header.token_len == 7 && token_begin[0] == 'p' && token_begin[1] == 'r' && token_begin[2] == 'i' && token_begin[3] == 'v' && token_begin[4] == 'a' && token_begin[5] == 't' && token_begin[6] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_PRIVATE ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 't' && token_begin[1] == 'h' && token_begin[2] == 'i' && token_begin[3] == 's' )
				token_dataunit_header.token_type = TOKEN_TYPE_THIS ;
			else if( token_dataunit_header.token_len == 9 && token_begin[0] == 'i' && token_begin[1] == 'n' && token_begin[2] == 't' && token_begin[3] == 'e' && token_begin[4] == 'r' && token_begin[5] == 'c' && token_begin[6] == 'e' && token_begin[7] == 'p' && token_begin[8] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_INTERCEPT ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'b' && token_begin[1] == 'e' && token_begin[2] == 'f' && token_begin[3] == 'o' && token_begin[4] == 'r' && token_begin[5] == 'e' )
				token_dataunit_header.token_type = TOKEN_TYPE_BEFORE ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'a' && token_begin[1] == 'f' && token_begin[2] == 't' && token_begin[3] == 'e' && token_begin[4] == 'r' )
				token_dataunit_header.token_type = TOKEN_TYPE_AFTER ;
			else if( token_dataunit_header.token_len == 3 && token_begin[0] == 's' && token_begin[1] == 'e' && token_begin[2] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_SET ;
			else if( token_dataunit_header.token_len == 3 && token_begin[0] == 'g' && token_begin[1] == 'e' && token_begin[2] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_GET ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'd' && token_begin[1] == 'e' && token_begin[2] == 'f' && token_begin[3] == 'e' && token_begin[4] == 'r' )
				token_dataunit_header.token_type = TOKEN_TYPE_DEFER ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'r' && token_begin[1] == 'e' && token_begin[2] == 't' && token_begin[3] == 'u' && token_begin[4] == 'r' && token_begin[5] == 'n' )
				token_dataunit_header.token_type = TOKEN_TYPE_RETURN ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 'e' && token_begin[1] == 'x' && token_begin[2] == 'i' && token_begin[3] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_EXIT ;
			else if( token_dataunit_header.token_len == 4 && token_begin[0] == 's' && token_begin[1] == 'y' && token_begin[2] == 'n' && token_begin[3] == 'c' )
				token_dataunit_header.token_type = TOKEN_TYPE_SYNC ;
			else if( token_dataunit_header.token_len == 6 && token_begin[0] == 'a' && token_begin[1] == 't' && token_begin[2] == 'o' && token_begin[3] == 'm' && token_begin[4] == 'i' && token_begin[5] == 'c' )
				token_dataunit_header.token_type = TOKEN_TYPE_ATOMIC ;
			else if( token_dataunit_header.token_len == 5 && token_begin[0] == 'c' && token_begin[1] == 'o' && token_begin[2] == 'n' && token_begin[3] == 's' && token_begin[4] == 't' )
				token_dataunit_header.token_type = TOKEN_TYPE_CONST ;
		}
		
#define IF_MATCH_IDENTIFICATION_WITH_CHARSET(_charset_,_literal2_) if( token_dataunit_header.token_len == sizeof(CHARSET_##_charset_##_literal2_)-1 && memcmp( token_begin , CHARSET_##_charset_##_literal2_ , sizeof(CHARSET_##_charset_##_literal2_)-1 ) == 0 )
#define ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET(_charset_,_literal2_) else IF_MATCH_IDENTIFICATION_WITH_CHARSET(_charset_,_literal2_)
#define MATCH_ALL_CHARSET_ALIAS(_charset_) \
	IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_TRUE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_TRUE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_FALSE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_FALSE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_NULL ) \
		token_dataunit_header.token_type = TOKEN_TYPE_NULL ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_INCLUDE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_INCLUDE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_NEW ) \
		token_dataunit_header.token_type = TOKEN_TYPE_NEW ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_IF ) \
		token_dataunit_header.token_type = TOKEN_TYPE_IF ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_ELSE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_ELSE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_SWITCH ) \
		token_dataunit_header.token_type = TOKEN_TYPE_SWITCH ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_CASE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_CASE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_DEFAULT ) \
		token_dataunit_header.token_type = TOKEN_TYPE_DEFAULT ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_DO ) \
		token_dataunit_header.token_type = TOKEN_TYPE_DO ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_WHILE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_WHILE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_FOR ) \
		token_dataunit_header.token_type = TOKEN_TYPE_FOR ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_IN ) \
		token_dataunit_header.token_type = TOKEN_TYPE_IN ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_FOREACH ) \
		token_dataunit_header.token_type = TOKEN_TYPE_FOREACH ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_CONTINUE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_CONTINUE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_BREAK ) \
		token_dataunit_header.token_type = TOKEN_TYPE_BREAK ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_IMPORT ) \
		token_dataunit_header.token_type = TOKEN_TYPE_IMPORT ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_FUNCTION ) \
		token_dataunit_header.token_type = TOKEN_TYPE_FUNCTION ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_OBJECT ) \
		token_dataunit_header.token_type = TOKEN_TYPE_OBJECT ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_EXTENDS ) \
		token_dataunit_header.token_type = TOKEN_TYPE_EXTENDS ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_INTERFACE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_INTERFACE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_IMPLEMENTS ) \
		token_dataunit_header.token_type = TOKEN_TYPE_IMPLEMENTS ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_PUBLIC ) \
		token_dataunit_header.token_type = TOKEN_TYPE_PUBLIC ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_PRIVATE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_PRIVATE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_THIS ) \
		token_dataunit_header.token_type = TOKEN_TYPE_THIS ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_INTERCEPT ) \
		token_dataunit_header.token_type = TOKEN_TYPE_INTERCEPT ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_BEFORE ) \
		token_dataunit_header.token_type = TOKEN_TYPE_BEFORE ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_AFTER ) \
		token_dataunit_header.token_type = TOKEN_TYPE_AFTER ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_SET ) \
		token_dataunit_header.token_type = TOKEN_TYPE_SET ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_GET ) \
		token_dataunit_header.token_type = TOKEN_TYPE_GET ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_RETURN ) \
		token_dataunit_header.token_type = TOKEN_TYPE_RETURN ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_EXIT ) \
		token_dataunit_header.token_type = TOKEN_TYPE_EXIT ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_SYNC ) \
		token_dataunit_header.token_type = TOKEN_TYPE_SYNC ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_ATOMIC ) \
		token_dataunit_header.token_type = TOKEN_TYPE_ATOMIC ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_CONST ) \
		token_dataunit_header.token_type = TOKEN_TYPE_CONST ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_TRY ) \
		token_dataunit_header.token_type = TOKEN_TYPE_TRY ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_CATCH ) \
		token_dataunit_header.token_type = TOKEN_TYPE_CATCH ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_UNCATCH ) \
		token_dataunit_header.token_type = TOKEN_TYPE_UNCATCH ; \
	ELSE_IF_MATCH_IDENTIFICATION_WITH_CHARSET( _charset_,_FINALLY ) \
		token_dataunit_header.token_type = TOKEN_TYPE_FINALLY ; \
			
		TEST_RUNTIME_DEBUG( rt )
		{
			TOKENTYPE_TO_STRING( token_dataunit_header.token_type , _zlang_token_type_str )
			PRINT_TABS_AND_FORMAT( rt , "TokenWord[%s:%"PRIi32",%"PRIi32"] type[%s] word[%.*s]" , token_dataunit_header.source_filename,token_dataunit_header.source_row,token_dataunit_header.source_col , _zlang_token_type_str , (int)token_dataunit_header.token_len,token_begin );
		}
		
		if( token_dataunit_header.token_type == TOKEN_TYPE_IDENTIFICATION )
		{
			if( rt->charset == ZLANG_CHARSET_GB18030 )
			{
				MATCH_ALL_CHARSET_ALIAS( GB18030 )
			}
			else if( rt->charset == ZLANG_CHARSET_UTF8 )
			{
				MATCH_ALL_CHARSET_ALIAS( UTF8 )
			}

			TEST_RUNTIME_DEBUG( rt )
			{
				TOKENTYPE_TO_STRING( token_dataunit_header.token_type , _zlang_token_type_str )
				PRINT_TABS_AND_FORMAT( rt , "MATCH_ALL_CHARSET_ALIAS[%s:%"PRIi32",%"PRIi32"] type[%s] word[%.*s]" , token_dataunit_header.source_filename,token_dataunit_header.source_row,token_dataunit_header.source_col , _zlang_token_type_str , (int)token_dataunit_header.token_len,token_begin );
			}
		}
		
		if( charset_flag == 1 )
		{
			if( token_dataunit_header.token_type == TOKEN_TYPE_DOUBLE_QUOTES_STRING )
			{
				char	charset[ 64 ] ;
				char	*p = NULL ;
				
				memset( charset , 0x00 , sizeof(charset) );
				snprintf( charset , sizeof(charset)-1 , "%.*s" , token_dataunit_header.token_len,token_begin );
				p = setlocale( LC_ALL , charset ) ;
				if( p == NULL )
				{
					SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_CHARSET , "charset[%s] invalid" , charset )
					return ZLANG_ERROR_CHARSET;
				}
				rt->full_charset_str = ZLSTRNDUP( token_begin , (size_t)(token_dataunit_header.token_len) ) ;
				if( STRISTR( charset , "GB2312" ) || STRISTR( charset , "GBK" ) || STRISTR( charset , "GB18030" ) )
				{
					rt->charset_str = WBYTE_CHARSET_GB18030_STR ;
					rt->charset = ZLANG_CHARSET_GB18030 ;
				}
				else if( STRISTR( charset , "UTF8" ) || STRISTR( charset , "UTF-8" ) )
				{
					rt->charset_str = WBYTE_CHARSET_UTF8_STR ;
					rt->charset = ZLANG_CHARSET_UTF8 ;
				}
			}
			else
			{
				SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_TOKEN , "expect quotes-string after 'charset'" )
				return ZLANG_ERROR_TOKEN;
			}
			
			charset_flag = 0 ;
		}
		else if( token_dataunit_header.token_type == TOKEN_TYPE_CHARSET )
		{
			charset_flag = 1 ;
		}
		else if( include_flag == 1 )
		{
			if( token_dataunit_header.token_type == TOKEN_TYPE_DOUBLE_QUOTES_STRING )
			{
				include_file = (struct ZlangIncludeFile *)ZLMALLOC( sizeof(struct ZlangIncludeFile) ) ;
				if( include_file == NULL )
				{
					SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_ALLOC , "alloc memory for include file" )
					return ZLANG_ERROR_ALLOC;
				}
				memset( include_file , 0x00 , sizeof(struct ZlangIncludeFile) );
				snprintf( include_file->include_filename , sizeof(include_file->include_filename) , "%s%c%.*s" , rt->first_z_file_pathname , DIRECTORY_SEPARATOR_CHAR , (int)(MIN(sizeof(include_file->include_filename)-1,token_dataunit_header.token_len)),token_begin );
				list_add_tail( & (include_file->this_include_file) , & (rt->include_files_list) );
				
				nret = LexicalAnalysisSourceFile( include_file->include_filename , rt ) ;
				if( nret )
				{
					return nret;
				}
			}
			else
			{
				SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_TOKEN , "expect quotes-string after 'include'" )
				return ZLANG_ERROR_TOKEN;
			}
			
			include_flag = 0 ;
		}
		else if( token_dataunit_header.token_type == TOKEN_TYPE_INCLUDE )
		{
			include_flag = 1 ;
		}
		else
		{
			do
			{
				token_dataunit = AddAppendonlyDataUnit( rt->curr_token_datapage->datapage , & token_dataunit_header , token_begin , token_dataunit_header.token_len ) ;
				if( token_dataunit == NULL )
				{
					if( GetAppendonlyDataPageLastError() == AODATAPAGE_WARN_SPACE_ISNOT_ENOUGH )
					{
						char		*datapage = NULL ;
						
						datapage = CreateAppendonlyDataPage( ZLANG_TOKEN_DATAPAGE_MAGIC , ZLANG_TOKEN_DATAPAGE_SIZE , sizeof(struct ZlangTokenDataPageHeader) , sizeof(struct ZlangTokenDataUnitHeader) , 1 ) ;
						if( datapage == NULL )
						{
							SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_ALLOC , "create token appendonly-datapage failed" )
							return ZLANG_ERROR_ALLOC;
						}
						token_datapage = GetAppendonlyDataPageUserHeader(datapage) ;
						token_datapage->datapage = datapage ;
						list_add_tail( & (token_datapage->this_token_datapage) , & (rt->token_datapage_list) );
						rt->curr_token_datapage = token_datapage ;
					}
					else
					{
						return ZLANG_ERROR_ADD_TOKEN;
					}
				}
			}
			while( token_dataunit == NULL );
			
			token_dataunit_header.source_col += token_dataunit_header.token_len ;
#if 0
			if( token_dataunit_header.token_type == TOKEN_TYPE_DOUBLE_QUOTES_STRING || token_dataunit_header.token_type == TOKEN_TYPE_SINGLE_QUOTES_STRING || token_dataunit_header.token_type == TOKEN_TYPE_BACK_QUOTES_STRING )
				token_dataunit_header.source_col += 2 ;
			if( token_dataunit_header.token_type == TOKEN_TYPE_BACK_QUOTES_STRING )
				token_dataunit_header.token_type = TOKEN_TYPE_IDENTIFICATION ;
#endif
		}
	}
	
	TEST_RUNTIME_DEBUG_THEN_PRINT( rt , "leave LexicalAnalysisSource filename[%s]" , source_filename )
	
	return 0;
}

void DebugLexicalAnalysis( struct ZlangRuntime *rt )
{
	struct ZlangSourceFileDataPageHeader	*source_file_datapage_header = NULL ;
	char					*source_file_dataunit = NULL ;
	char					*source_file_dataunit_body = NULL ;
	uint64_t				source_file_dataunit_body_len ;
	struct ZlangTokenDataPageHeader		*token_datapage_header = NULL ;
	char					*token_dataunit = NULL ;
	struct ZlangTokenDataUnitHeader		*token_dataunit_header = NULL ;
	char					*token_dataunit_body = NULL ;
	
	PRINT_TABS_AND_FORMAT( rt , "--- DebugLexicalAnalysis ---" );
	
	list_for_each_entry( source_file_datapage_header , & (rt->source_file_datapage_list) , struct ZlangSourceFileDataPageHeader , this_source_file_datapage )
	{
		PRINT_TABS_AND_FORMAT( rt , "source_file_datapage->datapage[%p]" , source_file_datapage_header->datapage );
		
		source_file_dataunit = NULL ;
		for( ; ; )
		{
			source_file_dataunit = TravelAppendonlyDataUnit( source_file_datapage_header->datapage , source_file_dataunit , NULL , & source_file_dataunit_body , & source_file_dataunit_body_len ) ;
			if( source_file_dataunit == NULL )
			{
				if( GetAppendonlyDataPageLastError() != AODATAPAGE_INFO_TRAVEL_OUT )
				{
					PRINT_TABS_AND_FORMAT( rt , "TravelDataUnit source_file_dataunit failed[%d]" , GetAppendonlyDataPageLastError() );
				}
				
				break;
			}
			
			PRINT_TABS_AND_FORMAT( rt , "    source_file[%.*s]" , (int)(source_file_dataunit_body_len),source_file_dataunit_body );
			
		}
	}
	
	list_for_each_entry( token_datapage_header , & (rt->token_datapage_list) , struct ZlangTokenDataPageHeader , this_token_datapage )
	{
		PRINT_TABS_AND_FORMAT( rt , "token_datapage[%p]->datapage[%p]" , token_datapage_header , token_datapage_header->datapage );
		
		token_dataunit = NULL ;
		for( ; ; )
		{
			token_dataunit = TravelAppendonlyDataUnit( token_datapage_header->datapage , token_dataunit , (void**) & token_dataunit_header , & token_dataunit_body , NULL ) ;
			if( token_dataunit == NULL )
			{
				if( GetAppendonlyDataPageLastError() != AODATAPAGE_INFO_TRAVEL_OUT )
				{
					PRINT_TABS_AND_FORMAT( rt , "TravelDataUnit token_dataunit failed[%d]" , GetAppendonlyDataPageLastError() );
				}
				
				break;
			}
			
			{
				TOKENTYPE_TO_STRING( token_dataunit_header->token_type , _zlang_token_type_str )
				PRINT_TABS_AND_FORMAT( rt , "    [%p] [%p][%s:%d,%d] token[%s][%.*s] p1[%p] p2[%p] n3[%d]" , token_dataunit , token_dataunit_header , token_dataunit_header->source_filename,(int)(token_dataunit_header->source_row),(int)(token_dataunit_header->source_col) , _zlang_token_type_str , (int)(token_dataunit_header->token_len),token_dataunit_body , token_dataunit_header->p1,token_dataunit_header->p2 , token_dataunit_header->n3 );
			}
		}
	}
	
	return;
}

int PeekToken( struct ZlangRuntime *rt , struct ZlangTokenDataUnitHeader **token_dataunit_header , char **token_dataunit_body )
{
	if( rt->travel_token_datapage_header == NULL )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_TRAVEL_NULL_ADDRESS , "peek null address" )
		return ZLANG_ERROR_TRAVEL_NULL_ADDRESS;
	}
	
	if( & (rt->travel_token_datapage_header->this_token_datapage) == & (rt->token_datapage_list) )
		return ZLANG_INFO_TRAVEL_TOKEN_OUT;
	
_GOTO_RETRY_TRAVELAPPENDONLYDATAUNIT :
	rt->travel_token_dataunit = PeekAppendonlyDataUnit( rt->travel_token_datapage_header->datapage , rt->travel_token_dataunit , (void**) token_dataunit_header , token_dataunit_body , NULL ) ;
	if( rt->travel_token_dataunit == NULL )
	{
		if( GetAppendonlyDataPageLastError() != AODATAPAGE_INFO_TRAVEL_OUT )
		{
			SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_INTERNAL , "peek token failed" )
			return ZLANG_ERROR_INTERNAL;
		}
		
		rt->travel_token_datapage_header = list_next_entry( rt->travel_token_datapage_header , struct ZlangTokenDataPageHeader , this_token_datapage ) ;
		if( & (rt->travel_token_datapage_header->this_token_datapage) == & (rt->token_datapage_list) )
			return ZLANG_INFO_TRAVEL_TOKEN_OUT;
		
		goto _GOTO_RETRY_TRAVELAPPENDONLYDATAUNIT;
	}
	
	return 0;
}

int NextToken( struct ZlangRuntime *rt )
{
	if( rt->travel_token_datapage_header == NULL )
	{
		SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_TRAVEL_NULL_ADDRESS , "next null address" )
		return ZLANG_ERROR_TRAVEL_NULL_ADDRESS;
	}
	
	if( & (rt->travel_token_datapage_header->this_token_datapage) == & (rt->token_datapage_list) )
		return ZLANG_INFO_TRAVEL_TOKEN_OUT;
	
_GOTO_RETRY_TRAVELAPPENDONLYDATAUNIT :
	rt->travel_token_dataunit = NextAppendonlyDataUnit( rt->travel_token_datapage_header->datapage , rt->travel_token_dataunit ) ;
	if( rt->travel_token_dataunit == NULL )
	{
		if( GetAppendonlyDataPageLastError() != AODATAPAGE_INFO_TRAVEL_OUT )
		{
			SET_RUNTIME_ERROR( rt , RUNTIME_ERROR , ZLANG_ERROR_INTERNAL , "peek token failed" )
			return ZLANG_ERROR_INTERNAL;
		}
		
		rt->travel_token_datapage_header = list_next_entry( rt->travel_token_datapage_header , struct ZlangTokenDataPageHeader , this_token_datapage ) ;
		if( & (rt->travel_token_datapage_header->this_token_datapage) == & (rt->token_datapage_list) )
			return ZLANG_INFO_TRAVEL_TOKEN_OUT;
		
		goto _GOTO_RETRY_TRAVELAPPENDONLYDATAUNIT;
	}
	
	return 0;
}

int TravelToken( struct ZlangRuntime *rt , struct ZlangTokenDataUnitHeader **token_dataunit_header , char **token_dataunit_body )
{
	int		nret = 0 ;
	
	nret = PeekToken( rt , token_dataunit_header , token_dataunit_body ) ;
	if( nret )
		return nret;
	
	nret = NextToken( rt ) ;
	if( nret )
		return nret;
	
	return 0;
}

